#ifndef __PAGE_LIB__
#define __PAGE_LIB__

#include "../include/simhash/include/Simhasher.hpp"
#include"RssTest.h"
#include<unistd.h>
#include <bitset>
#include<sstream>
#include <cmath>
#include<iostream>
#include<string>
#include <fstream>
#include <vector>
using std::string;
using std::endl;
using std::cout;
using std::vector;
using std::ofstream;
using namespace simhash;

//从_rss里面得数据
struct textPrep{
    int id;
    string text;
};
//得到id和world
struct keyWorld{
    int id;
    string world;
};

class PageLibPreprocessor{

public:
    PageLibPreprocessor(vector<RssiTeam> &RSS);
    ~PageLibPreprocessor();
    void Run(const string &filename); //启动！
    void InvertIndexTable();//打印权重信息
private:
    void ReadRss();
    //单个单词权重计算
    double calculateIDF(int tf,int totalDocuments, int documentFrequency);
    //归一化处理
    void NormaLization();
    void Jiebatest(const textPrep &t1);
    //进行分词操作
    void WorldKey();

private:
    //从RSS里面得到的库
    vector<RssiTeam> _RSS;
    //得到只有文本的库
    vector<textPrep> _prepText;
    //分出id 和world 要用到jieba分词
    vector<keyWorld> _worldKey;
    //单词 频率 出现的id
    unordered_map <string,pair<int,set<int>>> _keyIndex;
    //临时存储id string world 
    unordered_map <string,vector<pair<int,double>>> _inverIntexTable;
    //JIEBA jieba;
};
#endif
